sudo auditctl -w /data/solr7 -k whodeletedit -p wa

# In case auditctl is not installed:

sudo apt install -y auditd audispd-plugins

# And then execute the first command of this file again. It will keep track of all things that happen in that folder, in particular write and access, and tag it with "whodeletedit", for easy grepping later.

docker container stop solr7-localhost

docker container stop zk_solr

sudo rm -rf /var/lib/zookeeper/data/version-2/

docker run -d --name zk_solr -v /var/lib/zookeeper/data:/data -v /var/lib/zookeeper/datalog:/datalog -v /etc/zookeeper/zoo.cfg:/conf/zoo.cfg -p 127.0.0.1:2181:2181/tcp -p 127.0.0.1:2888:2888/tcp -p 127.0.0.1:3888:3888/tcp zookeeper:3.4.13

export ZK_H=`docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' zk_solr`

docker run --rm --name solr7_chroot solr:7.6.0-alpine bin/solr zk mkroot /solr -z $ZK_H:2181

docker run -d --name solr7-localhost -v /data/solr7/localhost/1/:/opt/solr/server/solr -e SOLR_PORT=8983 -e SOLR_HOST=localhost -e ZK_HOST=$ZK_H:2181/solr -e ZK_CLIENT_TIMEOUT=30000 solr:7.6.0-alpine

# wait a bit, check output of "ls -lah /data/solr7/localhost/1/" and you'll see the directories disappears. You can also track the logs of the solr7 container to see it complain about the cores.
# Then it can be checked with this:

sudo ausearch -i -k whodeletedit | grep -C 6 "DELETE"

# Part of the output:
type=PROCTITLE msg=audit(04/11/2019 01:03:32.642:2747096) : proctitle=/usr/lib/jvm/java-1.8-openjdk/jre/bin/java -server -Xms512m -Xmx512m -XX:NewRatio=3 -XX:SurvivorRatio=4 -XX:TargetSurvivorRatio= 
type=PATH msg=audit(04/11/2019 01:03:32.642:2747096) : item=1 name=/opt/solr/server/solr/example-DIH_shard2_replica_n4/core.properties inode=10485773 dev=08:16 mode=file,644 ouid=solr ogid=solr rdev=00:00 nametype=DELETE cap_fp=none cap_fi=none cap_fe=0 cap_fver=0 
type=PATH msg=audit(04/11/2019 01:03:32.642:2747096) : item=0 name=/opt/solr/server/solr/example-DIH_shard2_replica_n4/ inode=10485770 dev=08:16 mode=dir,755 ouid=solr ogid=solr rdev=00:00 nametype=PARENT cap_fp=none cap_fi=none cap_fe=0 cap_fver=0 
type=CWD msg=audit(04/11/2019 01:03:32.642:2747096) : cwd=/opt/solr/server 
type=SYSCALL msg=audit(04/11/2019 01:03:32.642:2747096) : arch=x86_64 syscall=unlink success=yes exit=0 a0=0x55813719b000 a1=0x55813750a000 a2=0x7f24cf1a440c a3=0x43 items=2 ppid=10694 pid=10718 auid=unset uid=solr gid=solr euid=solr suid=solr fsuid=solr egid=solr sgid=solr fsgid=solr tty=(none) ses=unset comm=java exe=/usr/lib/jvm/java-1.8-openjdk/jre/bin/java subj==docker-default (enforce) key=whodeletedit 
----
type=PROCTITLE msg=audit(04/11/2019 01:03:32.642:2747097) : proctitle=/usr/lib/jvm/java-1.8-openjdk/jre/bin/java -server -Xms512m -Xmx512m -XX:NewRatio=3 -XX:SurvivorRatio=4 -XX:TargetSurvivorRatio= 
type=PATH msg=audit(04/11/2019 01:03:32.642:2747097) : item=1 name=/opt/solr/server/solr/example-DIH_shard2_replica_n4 inode=10485770 dev=08:16 mode=dir,755 ouid=solr ogid=solr rdev=00:00 nametype=DELETE cap_fp=none cap_fi=none cap_fe=0 cap_fver=0 
type=PATH msg=audit(04/11/2019 01:03:32.642:2747097) : item=0 name=/opt/solr/server/solr/ inode=10485764 dev=08:16 mode=dir,755 ouid=solr ogid=root rdev=00:00 nametype=PARENT cap_fp=none cap_fi=none cap_fe=0 cap_fver=0 
type=CWD msg=audit(04/11/2019 01:03:32.642:2747097) : cwd=/opt/solr/server 
type=SYSCALL msg=audit(04/11/2019 01:03:32.642:2747097) : arch=x86_64 syscall=unlink success=no exit=EISDIR(Is a directory) a0=0x55813707f3a0 a1=0x55813750a000 a2=0x7f24cf1a440c a3=0x33 items=2 ppid=10694 pid=10718 auid=unset uid=solr gid=solr euid=solr suid=solr fsuid=solr egid=solr sgid=solr fsgid=solr tty=(none) ses=unset comm=java exe=/usr/lib/jvm/java-1.8-openjdk/jre/bin/java subj==docker-default (enforce) key=whodeletedit 
----
type=PROCTITLE msg=audit(04/11/2019 01:03:32.642:2747098) : proctitle=/usr/lib/jvm/java-1.8-openjdk/jre/bin/java -server -Xms512m -Xmx512m -XX:NewRatio=3 -XX:SurvivorRatio=4 -XX:TargetSurvivorRatio= 
type=PATH msg=audit(04/11/2019 01:03:32.642:2747098) : item=1 name=/opt/solr/server/solr/example-DIH_shard2_replica_n4 inode=10485770 dev=08:16 mode=dir,755 ouid=solr ogid=solr rdev=00:00 nametype=DELETE cap_fp=none cap_fi=none cap_fe=0 cap_fver=0 
type=PATH msg=audit(04/11/2019 01:03:32.642:2747098) : item=0 name=/opt/solr/server/solr/ inode=10485764 dev=08:16 mode=dir,755 ouid=solr ogid=root rdev=00:00 nametype=PARENT cap_fp=none cap_fi=none cap_fe=0 cap_fver=0 
type=CWD msg=audit(04/11/2019 01:03:32.642:2747098) : cwd=/opt/solr/server 
type=SYSCALL msg=audit(04/11/2019 01:03:32.642:2747098) : arch=x86_64 syscall=rmdir success=yes exit=0 a0=0x55813707f3a0 a1=0x55813750a000 a2=0x7f24cf1a440c a3=0x33 items=2 ppid=10694 pid=10718 auid=unset uid=solr gid=solr euid=solr suid=solr fsuid=solr egid=solr sgid=solr fsgid=solr tty=(none) ses=unset comm=java exe=/usr/lib/jvm/java-1.8-openjdk/jre/bin/java subj==docker-default (enforce) key=whodeletedit 
----
type=PROCTITLE msg=audit(04/11/2019 01:03:32.642:2747086) : proctitle=/usr/lib/jvm/java-1.8-openjdk/jre/bin/java -server -Xms512m -Xmx512m -XX:NewRatio=3 -XX:SurvivorRatio=4 -XX:TargetSurvivorRatio= 
type=PATH msg=audit(04/11/2019 01:03:32.642:2747086) : item=1 name=/opt/solr/server/solr/example-DIH_shard2_replica_n4/data/snapshot_metadata inode=10485778 dev=08:16 mode=dir,755 ouid=solr ogid=solr rdev=00:00 nametype=DELETE cap_fp=none cap_fi=none cap_fe=0 cap_fver=0 
type=PATH msg=audit(04/11/2019 01:03:32.642:2747086) : item=0 name=/opt/solr/server/solr/example-DIH_shard2_replica_n4/data/ inode=10485775 dev=08:16 mode=dir,755 ouid=solr ogid=solr rdev=00:00 nametype=PARENT cap_fp=none cap_fi=none cap_fe=0 cap_fver=0 
type=CWD msg=audit(04/11/2019 01:03:32.642:2747086) : cwd=/opt/solr/server 
type=SYSCALL msg=audit(04/11/2019 01:03:32.642:2747086) : arch=x86_64 syscall=unlink success=no exit=EISDIR(Is a directory) a0=0x55813707f3a0 a1=0x55813750a000 a2=0x7f24cf1a440c a3=0x4a items=2 ppid=10694 pid=10718 auid=unset uid=solr gid=solr euid=solr suid=solr fsuid=solr egid=solr sgid=solr fsgid=solr tty=(none) ses=unset comm=java exe=/usr/lib/jvm/java-1.8-openjdk/jre/bin/java subj==docker-default (enforce) key=whodeletedit 


# Direct your attention to "syscall=unlink" and "syscall=rmdir"

# Documentation:

https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/security_guide/sec-understanding_audit_log_files

http://man7.org/linux/man-pages/man2/unlink.2.html #  unlink, unlinkat - delete a name and possibly the file it refers to
http://man7.org/linux/man-pages/man2/rmdir.2.html # rmdir - delete a directory

# -----------
# pid in the ouput is 10718, looking that up gives(formatted):

sudo ps auxww | grep 10718
#solr     10718  0.5  0.3 2072740 218904 ?      Ssl  01:03   0:11 /usr/lib/jvm/java-1.8-openjdk/jre/bin/java -server -Xms512m -Xmx512m -XX:NewRatio=3 -XX:SurvivorRatio=4 -XX:TargetSurvivorRatio=90 -XX:MaxTenuringThreshold=8 -XX:+UseConcMarkSweepGC -XX:ConcGCThreads=4
# -XX:ParallelGCThreads=4 -XX:+CMSScavengeBeforeRemark -XX:PretenureSizeThreshold=64m -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=50 -XX:CMSMaxAbortablePrecleanTime=6000 -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:-OmitStackTraceInFastThrow
# -verbose:gc -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCTimeStamps -XX:+PrintTenuringDistribution -XX:+PrintGCApplicationStoppedTime -Xloggc:/opt/solr/server/logs/solr_gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=9 -XX:GCLogFileSize=20M
# -DzkClientTimeout=30000 -DzkHost=172.17.0.2:2181/solr -Dsolr.log.dir=/opt/solr/server/logs -Djetty.port=8983 -DSTOP.PORT=7983 -DSTOP.KEY=solrrocks -Dhost=localhost -Duser.timezone=UTC -Djetty.home=/opt/solr/server -Dsolr.solr.home=/opt/solr/server/solr -Dsolr.data.home= 
# -Dsolr.install.dir=/opt/solr -Dsolr.default.confdir=/opt/solr/server/solr/configsets/_default/conf -Dsun.net.inetaddr.ttl=60 -Dsun.net.inetaddr.negative.ttl=60 -Xss256k -Dsolr.jetty.https.port=8983 -jar start.jar --module=http

# Far as I can tell, this shows that it is the solr java process which is performing the deletes of the individual files and folders.


# To delete the auditctl rules:

sudo auditctl -D

# This deletes every single rule, so be careful if you already have some. If you already have some I'm going to assume you know how to delete the specific one that got added for this scenario.
